{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Data Explanation Benchmarking: Emotion Multiclass Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook demonstrates how to use the benchmark utility to benchmark the performance of an explainer for text data. In this demo, we showcase explanation performance for partition explainer on an Emotion Multiclass Classification model. The metrics used to evaluate are \"keep positive\" and \"keep negative\". The masker used is Text Masker.\n",
    "\n",
    "The new benchmark utility uses the new API with MaskedModel as wrapper around user-imported model and evaluates masked values of inputs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nlp\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy as sp\n",
    "import torch\n",
    "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
    "\n",
    "import shap\n",
    "import shap.benchmark as benchmark\n",
    "\n",
    "pd.set_option(\"display.max_columns\", None)\n",
    "pd.set_option(\"display.max_rows\", None)\n",
    "pd.set_option(\"max_colwidth\", None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Data and Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default\n"
     ]
    }
   ],
   "source": [
    "train, test = nlp.load_dataset(\"emotion\", split=[\"train\", \"test\"])\n",
    "\n",
    "data = {\"text\": train[\"text\"], \"emotion\": train[\"label\"]}\n",
    "\n",
    "data = pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"nateraw/bert-base-uncased-emotion\", use_fast=True)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\"nateraw/bert-base-uncased-emotion\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Class Label Mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set mapping between label and id\n",
    "id2label = model.config.id2label\n",
    "label2id = model.config.label2id\n",
    "labels = sorted(label2id, key=label2id.get)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Score Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(x):\n",
    "    tv = torch.tensor([tokenizer.encode(v, padding=\"max_length\", max_length=128, truncation=True) for v in x])\n",
    "    attention_mask = (tv != 0).type(torch.int64)\n",
    "    outputs = model(tv, attention_mask=attention_mask)[0].detach().numpy()\n",
    "    scores = (np.exp(outputs).T / np.exp(outputs).sum(-1)).T\n",
    "    val = sp.special.logit(scores)\n",
    "    return val"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Explainer Object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "explainers.Partition is still in an alpha state, so use with caution...\n"
     ]
    }
   ],
   "source": [
    "explainer = shap.Explainer(f, tokenizer, output_names=labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run SHAP Explanation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=42.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:   5%|█▋                               | 1/20 [00:00<?, ?it/s]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  15%|███▊                     | 3/20 [00:58<04:17, 15.12s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  20%|█████                    | 4/20 [01:28<05:12, 19.51s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  25%|██████▎                  | 5/20 [01:55<05:23, 21.59s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  30%|███████▌                 | 6/20 [02:20<05:17, 22.71s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  35%|████████▊                | 7/20 [02:45<05:03, 23.36s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  40%|██████████               | 8/20 [03:10<04:46, 23.86s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  45%|███████████▎             | 9/20 [03:35<04:27, 24.30s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  50%|████████████            | 10/20 [04:06<04:24, 26.43s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=30.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  55%|█████████████▏          | 11/20 [04:46<04:34, 30.47s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  60%|██████████████▍         | 12/20 [05:22<04:15, 31.91s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  65%|███████████████▌        | 13/20 [05:57<03:49, 32.84s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  70%|████████████████▊       | 14/20 [06:35<03:27, 34.52s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  75%|██████████████████      | 15/20 [07:15<03:01, 36.24s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  80%|███████████████████▏    | 16/20 [07:55<02:29, 37.32s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  85%|████████████████████▍   | 17/20 [08:34<01:53, 37.69s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  90%|█████████████████████▌  | 18/20 [09:06<01:12, 36.05s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer:  95%|██████████████████████▊ | 19/20 [09:42<00:36, 36.04s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Partition explainer: 100%|████████████████████████| 20/20 [10:24<00:00, 37.84s/it]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=48.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Partition explainer: 21it [10:52, 31.06s/it]                                      \n"
     ]
    }
   ],
   "source": [
    "shap_values = explainer(data[\"text\"][0:20])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Metrics (Sort Order & Perturbation Method)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "sort_order = \"positive\"\n",
    "perturbation = \"keep\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Benchmark Explainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEGCAYAAAB7DNKzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAsTUlEQVR4nO3dd3xc1Zn/8c+jLktyk+WCjS0bVwKmWLQApjkUQyCmLWwgoQSTRkhjIQskG0IWEpIQWPaX4DgFAjG9mN6N6biAwQ1wt9wk27LVrDZ6fn/MNSsbeTy2NLrSzPf9es1Lc+89M/e5kj3PnHPuOcfcHRERkV1JCzsAERHp3JQoREQkJiUKERGJSYlCRERiUqIQEZGYMsIOIBH69OnjxcXFYYchItJlzJkzZ6O7F7V2LCkTRXFxMbNnzw47DBGRLsPMVu7qmJqeREQkJiUKERGJSYlCRERiSso+itY0NjZSWlpKXV1d2KGklJycHAYNGkRmZmbYoYjIXkqZRFFaWkpBQQHFxcWYWdjhpAR3Z9OmTZSWljJ06NCwwxGRvZQyTU91dXUUFhYqSXQgM6OwsFC1OJEuLmUSBaAkEQL9zkW6vpRpehIRaQ/uTlOz09DUTFOz09wc3a5rjFDXGKG2IUJVXRNVdY1U1TdFy0WaaYw4DZFmIkH5zDQjOzON7Ix0cjPT6ZadTl5WBlkZaWSmp5GRbqTv4Ret9DTjgIE92v2alSg62BNPPMGkSZNYtGgRo0ePBmDGjBn87ne/4+mnn/683CWXXMIZZ5zBueeeS2NjIzfeeCOPPvooBQUFZGdn8/Of/5zTTjtth/e+6667+OMf/8jSpUspLy+nT58+rcZw6qmn8u6773LMMcfscE5354YbbuDhhx8mPT2d73znO/zgBz/gtttu4/777wegqamJRYsWUV5eTu/evQGIRCKUlJQwcODAHd5PpDNzd7bUNrJuax0bKusor6pnU00Dm2vq2VLbSFVdE5V10Z/V9U1U1TVRU99EfVOE5k66jE+f/Gxm3zCh3d9XiaKDTZs2jWOOOYZp06bxy1/+Mq7X3Hjjjaxbt4758+eTnZ3Nhg0beP31179Q7uijj+aMM87g+OOPj/l+11xzDbW1tdx999077P/HP/7B6tWrWbx4MWlpaZSVlX1e/pprrgHgqaee4vbbb/88SQDccccdjBkzhsrKyriuR6Qttn+jj36Db2ZbQ4Sahqbgm3xj8G0++qFeVR/9WVMf/bCvqW9ic00D5VX1bKxuoCHS/IX3z8lMo2duFt1zM+iek0lhfhbFffLIz84gLyudnMx0cjKj3/rT04z0NCMjzcjJTCc3K1o7KMjJpCAng/zsDLJb1BAy09PICF4TaXbqm5qpa4ywrTESXEeEhqZmGiPNNESa2dOF5bLS09vr17wDJYoOVF1dzZtvvslrr73GV7/61bgSRW1tLX/5y19Yvnw52dnZAPTr14/zzz//C2UPOeSQuOI46aSTmDFjxhf2/+lPf+Jf//oXaWnRrqu+fft+ocy0adO48MILP98uLS3lmWee4frrr+cPf/hDXOcX2W5bQ4QNlXWsr9zxW31FTQObaxrYuq2RyqAZpy74MK1rijbfxKtbVjr52dEP7bzsDHrlZTG8bwF9CrLoV5BD/x7RR1F+NoX5WXTL6piPxYx0IyM9jbzszv8x3PkjTIBfPrWAhWvb99vv/vt05xdf/VLMMk8++SSnnnoqI0eOpLCwkDlz5jBu3LiYr1myZAmDBw+me/fu7Rluq5YuXcqDDz7I448/TlFREXfeeScjRoz4/HhtbS3PP/88d9111+f7fvjDH/Lb3/6WqqqqhMcnnZe7U1HbyLqt26jc1vR5W311fWP0w35bE1u2NVBR28iW2gY2VjWwvrKOrdsav/BeGWlG77wsenXLokduJgN75tI9p4DcFt/mczL+73luVvSbfm5WOgU50VpAQU4meUGbf1qabqhoq5RMFGGZNm0aV199NQAXXHAB06ZNY9y4cbu8M6ij7xiqr68nJyeH2bNn89hjj3HZZZfxxhtvfH78qaee4uijj/682enpp5+mb9++jBs3rtUaiiSfxkgzqzfXsqy8hsXrK1m0vorF6yoprdhGfdMXm3G2S08zuudk0KtbFj27ZTK4sBuHD+1N/x459OueQ//uOfTvkU1RQQ7dczJ0t1wnk5KJYnff/BNh8+bNvPrqq3z88ceYGZFIBDPjtttuo7CwkIqKii+U79OnD8OHD2fVqlVUVlYmvFYxaNAgzj77bAAmTZrEpZdeusPxBx54YIdmp7feeovp06fz7LPPUldXR2VlJRdddBH33XdfQuOUxKqpb2L5xhpWbKph5aZaSitqKa3YxurNtayu2LZDs8++vXMZ0787J43pF3zY59CzW2b0Lp6sDPJzMuiRm0leVro+/LuwlEwUYXjkkUe4+OKLd+hAPu6443jjjTc44ogjWLt2LYsWLWLMmDGsXLmSefPmcfDBB9OtWzcuv/xyrr76au6++26ysrIoLy9nxowZnHfeee0a49e+9jVee+01hg4dyuuvv87IkSM/P7Z161Zef/31HZLALbfcwi233AL8351bShJdQ1OkmfWVdazYWMvyTTUsK69mSVn0sW7rjgMk++RnMbBXN740sAdnjN2HoX3yGFqUx4i++RTkaGqWVKBE0UGmTZvGtddeu8O+c845h2nTpjF+/Hjuu+8+Lr30Uurq6sjMzGTq1Kn06BG9H/rmm2/mhhtuYP/99ycnJ4e8vDxuuummL5zjzjvv5Le//S3r169n7NixTJw4kalTpzJ79mz+/Oc/M3XqVACOPfZYFi9eTHV1NYMGDeKvf/0rp5xyCtdddx1f//rXuf3228nPz/+8PMDjjz/OySefTF5eXgJ/S9JWTZFmNlTVs35rtGO4vLo++jN4lFXVsX5rHRur63e4xTM3M53hffM5clghw/vmM7RPHkMKu1FcmNclOlslsWxPb7/qCkpKSnznhYu2f1uXjqfffftqbnZKK7bxWVkVS8urWb6xhmXlNazeXMv6yrov3ONvBoV52RQVRB/9u2cHzUS50dpBnzz6dc9W01CKM7M57l7S2jF9VRDpBJqbnYra6O2gLQd3bQvuHlq7ZRvLNtawIkgK2xojn7+2MC96n/+RwwoZ2CuXfXrm0r97DkUF2fQtyKZ3XhYZ6Sk1W4+0MyUKkQ5Q1xhh1eZaVmysYdXmWtZuqWN95bboqOCtdZRV1dMUY2xAepoxqFcuxYV5HDG0kJH98hnRr4DhRfn06KZ+AkmslEoU7q7qdQdLxqbNWBqamlm2sZpPN1Tz6foqPtlQxacbqli1uZaWv4puWekM6JHDgB65HLVfH/p1j37779kti/zsDApyooPDto/07ZOfTVaGagUSjpRJFDk5OWzatElTjXeg7etR5OTkhB3Kbm2uaeD95Zt4b/lmVmysYUNlPWVV9dQ1RqIjaNPSyEy36DQMaWmkpRnN7rhDpMWEcDUNkc9vH01PM4b2yeOAgT04+5BBDC3Ko7iwG0N659E9V2MFpOtImUQxaNAgSktLKS8vDzuUlLJ9hbvOaOWmGp7+aB3PfLSOheuiI/VzMtPYryif/j1yGDuoB7lZ6TRFonMLNUWis4U2RpppdifNDDMj3QhGCUdHBg/vm8/IfgUMK8ojOyMxc++IdKTQEoWZ9QYeBIqBFcD57l6xi7LdgYXAE+7+/b05X2ZmplZZE9ydGZ+Wc+crn/HBqi0AHDq4J9ecMoojh/XmwIE91cQjspMwaxTXAa+4+61mdl2wfe0uyv4KmNlhkUlSenfZJn73wifMXlnBoF65/OfE0Zw+dh8G9swNOzSRTi3MRHEWcHzw/B5gBq0kCjMbB/QDngdavcdXJJa1W7Zx8zMLefbj9fTrns2vvnYA/1ayr2oOInEKM1H0c/d1wfP1RJPBDswsDfg9cBEQczUOM5sMTAYYPHhw+0YqXVJ9U4SpbyznrleX0OzOj78yksnjh5GTqX4DkT2R0ERhZi8D/Vs5dH3LDXd3M2vtPsrvAs+6e+nu7hBx9ynAFIiOzN67iCVZzPikjF8+tZDlG2s45Uv9uOH0/dm3d7ewwxLpkhKaKNx9l7UAM9tgZgPcfZ2ZDQDKWil2FHCsmX0XyAeyzKza3a9LUMjSxW2orOPnT87nhQUbGNonj3suO5zjRhaFHZZIlxZm09N04JvArcHPJ3cu4O5f3/7czC4BSpQkZFemz1vLjU/Mp74pwjWnjOJbxw7V7aki7SDMRHEr8JCZXQ6sBM4HMLMS4Nvu/q0QY5MupKyqjpueWsjTH63j4H178ofzD2JYUX7YYYkkjZSZPVaST21DE1NmLmPKzGU0Rpr54YSRXDl+mCbAE9kLmj1Wkkp1fRMPvL+KKTOXUVZVz+kHDuA/Th3FkEKtlSGSCEoU0mWUV9Xzj7eX8893VlJZ18QRQ3vzp4sOZdyQ3mGHJpLUlCik01uxsYYpbyzjkTmlNEaaOfVL/Zk8fhiHDO4VdmgiKUGJQjqtzTUN/Oa5xTw8ZzUZ6Wmcc+ggJo8fxtA+amIS6UhKFNLpNDc7D81eza3PL6a6rolLjx7KlccNo29B55+uXCQZKVFIp7JoXSU3PDGfOSsrOKy4Fzd/7UBG9S8IOyyRlKZEIZ1CdX0Tf3zpU/7+9gp65GZy27ljOXfcIC3uI9IJKFFI6N78bCPXPvoRa7Zs48LD9+U/ThlNr7yssMMSkYAShYSmur6J/352Ef96bxXD+uTxyLePoqRYt7qKdDZKFBKK1z8t5z8f+5i1W7dxxbFD+cnJozT9t0gnpUQhHWpLbQM3Pb2Qx+auYb+iaC1CA+ZEOjclCukQ7s6zH6/nF9Pns6W2katOHM73ThiuWoRIF6BEIQm3obKOG5+Yz4sLN3DgwB7ce9kR7L9P97DDEpE4KVFIwlTUNDDljWXc8/YKIs3Oz04bzeXHDNXsriJdjBKFtLtVm2p5YNYq7n1nJTUNTXx17D78+CsjKdbUGyJdkhKFtIu6xgjPzV/HQ7NKeWfZJszgtAP688MJIxnZTyOrRboyJQrZa+7O4vVVPDhrNY/NLaWyronBvbvx05NHcvahg9inZ27YIYpIO1CikD3i7ryzdBMvLdrAK4vKWLW5lqz0NE45oD8XHrYvRw4rJC1N026IJBMlConbO0s38ZvnF/Ph6i1kZ6Rx9PA+TB4/jIkHDqC3ptwQSVqhJAoz6w08CBQDK4Dz3b2ilXKDganAvoADE919RYcFKgCs3FTDjU8uYOan5QzokcNvzjmQMw8aSG6WxkCIpIKwahTXAa+4+61mdl2wfW0r5e4Ffu3uL5lZPtDckUGmOvfouhC/fGoh6WnGDaeP4aIjh2iQnEiKCStRnAUcHzy/B5jBTonCzPYHMtz9JQB3r+7A+FJeRU0D1z76ES8u3MCX9yvkd+cdpM5pkRQVVqLo5+7rgufrgX6tlBkJbDGzx4ChwMvAde4e6aAYU9aclRVc9a+5bKxu4IbTx3DZ0UPVQS2SwhKWKMzsZaB/K4eub7nh7m5m3kq5DOBY4BBgFdE+jUuAv+7ifJOByQCDBw/e67hTmbvz1zeXc+tzixnQM4fHvvtlDhjYI+ywRCRkCUsU7j5hV8fMbIOZDXD3dWY2AChrpVgp8KG7Lwte8wRwJLtIFO4+BZgCUFJS0lrikRjqmyJc+8hHPPHhWk7evx+3nXcQPXIzww5LRDqBsCbdmQ58M3j+TeDJVsrMAnqaWVGwfSKwsANiSzmbaxq4aOp7PPHhWn568kjuvnickoSIfC6sRHEr8BUz+wyYEGxjZiVmNhUg6Iv4KfCKmX0MGPCXkOJNWqs21TLp/73FvNKt/M+Fh/D9E0donWoR2UEondnuvgk4qZX9s4Fvtdh+CRjbgaGllNqGJr517yy2bmtk2hVHMm5Ir7BDEpFOSCOzU5S787PHPuazsmruvexwJQkR2SUtDJCi7nt3JU9+uJYfTxjJsSOKdv8CEUlZShQp6MPVW7jp6YWcOLov3ztheNjhiEgnp0SRYmobmrj6gQ/oW5DD7ecfrIF0IrJb6qNIMbc8u5hVm2uZdsWR9OimW2BFZPdUo0ghr39azj/fXcnlRw/lyGGFYYcjIl2EEkWK2FrbyH88Mo8RffP56Smjwg5HRLoQNT2lAHfnZ49/xKbqBqZ+4zBNEy4ie0Q1ihQwZeYynv14PdecMooDB2mSPxHZM0oUSe6tJRv5zfOLOf3AAUwePyzscESkC1KiSGKlFbV8/19z2a8on9+eO1ZzOInIXlGiSFINTc187/65NEWcuy8eR162uqNEZO/o0yNJ3frcYuaVbuXPFx3KsKL8sMMRkS5MNYok9OKC9fztreVc8uViTj1gQNjhiEgXp0SRZEoravnpw/M4cGAPfjZxdNjhiEgSUKJIIs3Nzo8e/BB3+N9/P5TsDI2XEJG2222iMLPz4tkn4Xt4zmpmrajgxq/uz+DCbmGHIyJJIp4axc/i3Cch2lzTwC3PLebw4t6cN25Q2OGISBLZ5V1PZnYaMBEYaGZ3tjjUHWhKdGCyZ259bhHVdU3cPOkAjZcQkXYV6/bYtcBs4ExgTov9VcCPEhmU7JnZKzbz0OxSvn3cfozsVxB2OCKSZHaZKNx9HjDPzO5393avQZhZb+BBoBhYAZzv7hWtlPstcDrRZrKXgKvd3ds7nq6qtqGJ6x77mIE9c/nBSVqtTkTaXzx9FJ+Z2bKdH+1w7uuAV9x9BPBKsL0DM/sycDQwFjgAOAw4rh3OnTR+8eQClpZX85tzxtItS+MnRaT9xfPJUtLieQ5wHtC7Hc59FnB88PweYAZw7U5lPDhnFmBAJrChHc6dFB6dU8rDc0r5wYnDOWZEn7DDEZEktdsahbtvavFY4+5/JNoU1Fb93H1d8Hw90K+Vc78DvAasCx4vuPui1t7MzCab2Wwzm11eXt4O4XVuS8qquOGJ+Rw5rDdXTxgZdjgiksR2W6Mws0NbbKYRrWHE1cZhZi8D/Vs5dH3LDXd3M/tCv4OZDQfGANvv93zJzI519zd2LuvuU4ApACUlJUndh1HXGOF7939At6x07rjgENLTdJeTiCROPB/4v2/xvImg4zmeN3f3Cbs6ZmYbzGyAu68zswFAWSvFJgHvunt18JrngKOALySKVPL7Fz/hkw1V/P3Sw+jXPSfscEQkye02Ubj7CQk693Tgm8Ctwc8nWymzCrjCzG4h2kdxHPDHBMXTJby7bBNT31zORUcO5oRRfcMOR0RSQDxTeBSa2Z1mNtfM5pjZHWZW2A7nvhX4ipl9BkwItjGzEjObGpR5BFgKfAzMA+a5+1PtcO4uqaqukZ88NI8hvbvxnxPHhB2OiKSIeJqeHgBmAucE218nOv5hl81K8XD3TcBJreyfDXwreB4BrmzLeZLJr55eyLqt23j421/WrbAi0mHi+bQZ4O6/arF9s5n9W6ICktYtWlfJQ7NLufK4YYwb0ivscEQkhcQz4O5FM7vAzNKCx/nAC4kOTHb0l5nL6JaVzneO2y/sUEQkxcSTKK4A/gXUB48HgCvNrMrMKhMZnESt3bKN6fPWcsFhg+nZLSvscEQkxcRz15NmmQvZ395cjgOXHzs07FBEJAXFc9fTK/Hsk8TYWtvItPdXceZB+zCwZ27Y4YhICoq1HkUO0A3oY2a9iI5jgOh6FAM7IDYB7ntvJTUNESaPHxZ2KCKSomI1PV0J/BDYB5jbYn8lcFcCY5JAfVOEv7+1guNGFjFmQPewwxGRFBVrPYo7gDvM7Cp3/58OjEkCz3y0jo3V9XxLfRMiEqJ4xlFsNbNv7LzT3e9NQDzSwj1vr2C/ojyOGa4pxEUkPPEkisNaPM8hOpp6LqBEkUAfrt7CvNKt3HTWl7QGtoiEKp7bY69quW1mPYmOpZAEuuftFeRnZ3D2oYN2X1hEJIHiGXC3sxpAjeYJVF5VzzMfrePccYPIz9acTiISrngWLnqK6JKkEE0s+wMPJTKoVPfA+6toiDRz8VFDwg5FRCSuPorftXjeBKx099IExZPyGiPN3P/eKo4d0Yf9ivLDDkdEJK6mp1VAQfBYpySRWK8sKmN9ZR3fOKo47FBERIDYI7O7A1OBcUQXDQI42MzmAJe7uyYETIBp769iQI8cThhVFHYoIiJA7BrFncBCYIS7n+3uZwP7EV1tTiOzE2D15lpmflbO+SX7kpG+N/cZiIi0v1h9FEe7+yUtd7i7AzcFy5dKO3to9moMOP+wfcMORUTkc3v7tVUjwNpZU6SZB2et5vhRfTVLrIh0KrESxdtm9nPbaViwmd0IvNOWk5rZeWa2wMyazawkRrlTzewTM1tiZte15Zyd3auLyyirqufCwweHHYqIyA5iJYqrgAOBJWb2aPBYChwEfL+N550PnA3M3FUBM0sH/hc4jejYjQvNbP82nrfTmvb+Kvp1z1Yntoh0OrFmj60EzjOz/Yh+UAMsdPelbT2puy8CdjeH0eHAEndfFpR9ADiLaAd7UlmzZRszPi3nqhOGqxNbRDqdeOZ6Wgq0OTnshYHA6hbbpcARuypsZpOByQCDB3et5psnPliDO5xXok5sEel8EjaRkJm9DPRv5dD17v5ke5/P3acAUwBKSkp8N8U7DXfn8Q/WcFhxL/bt3S3scEREviBhicLdJ7TxLdYALb9iDwr2JZUFaytZUlbNrycdEHYoIiKtijUyu3esF7r75vYPZwezgBFmNpRogrgA+PcEn7PDPTZ3DVnpaZxx4D5hhyIi0qpYNYo5RGeNba3H2YFhe3tSM5sE/A9QBDxjZh+6+ylmtg8w1d0nunuTmX0feAFIB/7m7gv29pydUVOkmenz1nLC6CJ6dMsMOxwRkVbFuuspYWtOuPvjwOOt7F8LTGyx/SzwbKLiCNubSzaysbqeSYdocSIR6bx2ey+mRV0UDLTDzAab2eGJDy35PfHBGnrkZnLCaI2dEJHOK56b9v8fcBT/1z9QRXQgnLRBTX0TLyzYwOljB5CdkR52OCIiuxTPXU9HuPuhZvYBgLtXmFlWguNKeq8sLmNbY4SvHTww7FBERGKKp0bRGEyn4QBmVgQ0JzSqFPDKog30zsti3JBeYYciIhJTPIniTqIdz33N7NfAm8B/JzSqJNcUaWbGJ+WcMKov6WmaiFdEOrd4pvC4P1jV7iSit8p+bftcTbJ3Zq+sYOu2RiaM6Rt2KCIiuxXvgLsyYFrLYx0w4C5pvbJoA1npaRw7Unc7iUjnF++Au8FARfC8J7AKSNg4i2T3yqIyjhjWm/zshM2gIiLSbnbZR+HuQ919GPAy8FV37+PuhcAZwIsdFWCyWVZezbKNNUwY0y/sUERE4hJPZ/aRwQhpANz9OeDLiQspub26uAyAE0erf0JEuoZ42j7WmtkNwH3B9teBtYkLKbm9vGgDo/oVaEpxEeky4qlRXEh08r7t8zP1DfbJHtpa28isFRWcpLudRKQLief22M3A1WZWEN306sSHlZxmflZOpNk5Sf0TItKFxDMp4IHB9B3zgQVmNsfMtMrOXnhv+SbystI5aFCPsEMREYlbPE1PdwM/dvch7j4E+AnBkqOyZ2Ytr+DQIb3ISI/n1y4i0jnE84mV5+6vbd9w9xlAXsIiSlJbahv4ZEMVhxfHXDhQRKTTieeup2XBWhT/DLYvApYlLqTkNHtFBQCHDVWiEJGuJZ4axWVE73p6LHgUBftkD8xauZnMdOPgfXuGHYqIyB6J566nCuAHHRBLUpu1fDNjB/UkJ1OLFIlI1xJrUsDpsV7o7mfu7UnN7Dzgv4AxwOHuPruVMvsC9wL9iM45NcXd79jbc4aprjHCx2u2cvkxw8IORURkj8WqURwFrCY6a+x7RCcEbC/zgbOJ3lG1K03AT9x9bjCGY46ZveTuC9sxjg7xwaotNEacw4dqkSIR6XpiJYr+wFeIjsL+d+AZYJq7L2jrSbevZ2G269zj7uuAdcHzKjNbBAwEulyimLViM2Ywbog6skWk64k1e2zE3Z93928CRwJLgBlm9v0Oiy5gZsXAIURrNl3OrBWbGdWvgB65mWGHIiKyx2J2ZptZNnA60VpFMf+3LOpumdnLRGslO7ve3Z+MN0AzywceBX7o7pUxyk0GJgMMHjw43rdPuKZIM3NXVnDOuEFhhyIisldidWbfCxwAPAv80t3n78kbu/uENsaGmWUSTRL3u/tjuznfFIIR4yUlJd7Wc7eXhesqqWmIcJgG2olIFxWrRnERUANcDfygRX+CEZ0csHsiA7PoCf8KLHL3PyTyXIk0a/tAOyUKEemiYvVRpLl7QfDo3uJR0NYkYWaTzKyU6J1Vz5jZC8H+fcxs+yJJRwMXAyea2YfBY2JbzhuGuSsrGNgzl/49csIORURkr4SyaLO7b1/bYuf9a4GJwfM3ad9bckMxZ2WFpu0QkS5N05gm0Not21hfWce4wT3DDkVEZK8pUSTQ3FXR/olDh2ignYh0XUoUCTRnZQU5mWmMGZDQfn8RkYRSokiguau2MHZQTzK1UJGIdGH6BEuQusYIC9ZsZZyanUSki1OiSJCP12ylqdk5dLAShYh0bUoUCTJ3ZbQj+xDd8SQiXZwSRYLMWVlBcWE3+uRnhx2KiEibKFEkgLszd9UWNTuJSFJQokiA1Zu3sbG6XuMnRCQpKFEkwOcD7VSjEJEkoESRAHNWVpCXlc6o/gVhhyIi0mZKFAkwa8VmDh3Si/S0Lj+noYiIEkV7q6xr5JMNVZRofWwRSRJKFO1s7soK3OGwYvVPiEhyUKJoZ7NXVJCeZhysgXYikiSUKNrZrBWb+dI+3emWFcqaUCIi7U6Joh01NDUzr3SL+idEJKkoUbSjBWu3UtfYrP4JEUkqShTtaPaK6EC7cUoUIpJEQkkUZnaemS0ws2YzK9lN2XQz+8DMnu6o+PbWrBWbGVLYjb4FOWGHIiLSbsKqUcwHzgZmxlH2amBRYsNpO3dnzsoK9U+ISNIJJVG4+yJ3/2R35cxsEHA6MDXxUbXN8o01bKppUP+EiCSdzt5H8UfgP4Dm3RU0s8lmNtvMZpeXlyc8sJ1t758oKVaNQkSSS8IShZm9bGbzW3mcFefrzwDK3H1OPOXdfYq7l7h7SVFRUZti3xvvLt9E77ws9ivK6/Bzi4gkUsJGhbn7hDa+xdHAmWY2EcgBupvZfe5+Uduja1/Nzc7rn5QzfkQfzDQRoIgkl07b9OTuP3P3Qe5eDFwAvNoZkwTAvNItbKpp4ITRfcMORUSk3YV1e+wkMysFjgKeMbMXgv37mNmzYcTUFq8uLiPN4LiRHd/kJSKSaKFMSOTujwOPt7J/LTCxlf0zgBkJD2wvvbq4jHFDetGzW1bYoYiItLtO2/TUVWyorGPB2ko1O4lI0lKiaKPXFpcBcKIShYgkKSWKNnp1cRn79MhhVD+tjy0iyUmJog3qmyK8uWQjJ47pq9tiRSRpKVG0wfvLN1PbEFGzk4gkNSWKNnhlURnZGWkcNaxP2KGIiCSMEsVeqmuM8MSHazhxdF9ys9LDDkdEJGGUKPbS9Hlr2VLbyDeOKg47FBGRhFKi2Avuzr3vrGBkv3yOHKbZYkUkuSlR7IUPVm9h/ppKLj6qWHc7iUjSU6LYC/e+vYKC7AzOPmRg2KGIiCScEsUeKq+q55mP13HOuEHkZYcyVZaISIdSothDD85aRWPEuejIIWGHIiLSIZQo9kB9U4R73lnJsSP6MLxvftjhiIh0CCWKPfDkB2spr6rnimOHhR2KiEiHUaKIU3OzM+WNZYwZ0J1jR2gktoikDiWKOL26uIwlZdVcOX6YbokVkZSiRBGnKTOXMbBnLqePHRB2KCIiHUqJIg5zV1Xw/orNXHbMUDLT9SsTkdQSyqeemZ1nZgvMrNnMSmKU62lmj5jZYjNbZGZHdWSc2015fRndczK44LB9wzi9iEiowvp6PB84G5i5m3J3AM+7+2jgIGBRogPb2ZKyal5YuJ6LjxqiAXYikpJC+eRz90VAzE5hM+sBjAcuCV7TADR0QHg7+PPrS8nOSOOyo4d29KlFRDqFztzgPhQoB/5uZh+Y2VQzy9tVYTObbGazzWx2eXl5uwSwZss2nvhgDRccNpjC/Ox2eU8Rka4mYYnCzF42s/mtPM6K8y0ygEOBP7n7IUANcN2uCrv7FHcvcfeSoqKidrgC+MvMZQBcMV4D7EQkdSWs6cndJ7TxLUqBUnd/L9h+hBiJor1trK5n2vurmHTIQAb2zO2o04qIdDqdtunJ3dcDq81sVLDrJGBhR53/728tpyHSzJXH7ddRpxQR6ZTCuj12kpmVAkcBz5jZC8H+fczs2RZFrwLuN7OPgIOB/+6I+CrrGrn37ZWcdkB/Tf4nIikvrLueHgceb2X/WmBii+0PgV2Os0iU+95dSVV9E989fnhHn1pEpNPptE1PYalrjPC3N5czfmQRBwzsEXY4IiKhU6LYyUOzV7OxuoHvHa++CRERUKLYQWOkmbtfX8a4Ib04fGjvsMMREekUlChamP7hWtZs2cb3TthPU4mLiASUKALNzc6fXl/K6P4FnDCqb9jhiIh0GprlLlDbGGHc4F6MH1mk2oSISAtKFIH87Ax+c+7YsMMQEel01PQkIiIxKVGIiEhMShQiIhKTEoWIiMSkRCEiIjEpUYiISExKFCIiEpMShYiIxGTuHnYM7c7MyoGVe/nyPsDGdgynK9A1J79Uu17QNe+pIe5e1NqBpEwUbWFms929wxdLCpOuOfml2vWCrrk9qelJRERiUqIQEZGYlCi+aErYAYRA15z8Uu16QdfcbtRHISIiMalGISIiMSlRiIhITCmbKMzsVDP7xMyWmNl1rRzPNrMHg+PvmVlxCGG2mziu98dmttDMPjKzV8xsSBhxtqfdXXOLcueYmZtZl7+VMp5rNrPzg7/1AjP7V0fH2N7i+Lc92MxeM7MPgn/fE8OIs72Y2d/MrMzM5u/iuJnZncHv4yMzO7TNJ3X3lHsA6cBSYBiQBcwD9t+pzHeBPwfPLwAeDDvuBF/vCUC34Pl3uvL1xnvNQbkCYCbwLlASdtwd8HceAXwA9Aq2+4Yddwdc8xTgO8Hz/YEVYcfdxmseDxwKzN/F8YnAc4ABRwLvtfWcqVqjOBxY4u7L3L0BeAA4a6cyZwH3BM8fAU6yrruY9m6v191fc/faYPNdYFAHx9je4vkbA/wK+A1Q15HBJUg813wF8L/uXgHg7mUdHGN7i+eaHegePO8BrO3A+Nqdu88ENscochZwr0e9C/Q0swFtOWeqJoqBwOoW26XBvlbLuHsTsBUo7JDo2l8819vS5US/kXRlu73moEq+r7s/05GBJVA8f+eRwEgze8vM3jWzUzssusSI55r/C7jIzEqBZ4GrOia00Ozp//fdymhTOJJ0zOwioAQ4LuxYEsnM0oA/AJeEHEpHyyDa/HQ80VrjTDM70N23hBlUgl0I/MPdf29mRwH/NLMD3L057MC6ilStUawB9m2xPSjY12oZM8sgWmXd1CHRtb94rhczmwBcD5zp7vUdFFui7O6aC4ADgBlmtoJoW+70Lt6hHc/fuRSY7u6N7r4c+JRo4uiq4rnmy4GHANz9HSCH6OR5ySqu/+97IlUTxSxghJkNNbMsop3V03cqMx34ZvD8XOBVD3qKuqDdXq+ZHQLcTTRJdPV2a9jNNbv7Vnfv4+7F7l5MtF/mTHefHU647SKef9dPEK1NYGZ9iDZFLevAGNtbPNe8CjgJwMzGEE0U5R0aZceaDnwjuPvpSGCru69ryxumZNOTuzeZ2feBF4jeNfE3d19gZjcBs919OvBXolXUJUQ7ji4IL+K2ifN6bwPygYeDPvtV7n5maEG3UZzXnFTivOYXgJPNbCEQAa5x965aU473mn8C/MXMfkS0Y/uSLvylDzObRjTZ9wn6XX4BZAK4+5+J9sNMBJYAtcClbT5nF/59iYhIB0jVpicREYmTEoWIiMSkRCEiIjEpUYiISExKFCIiEpMShXRJZhYxsw/NbL6ZPWxm3UKI4Xgz+/Iujv2Xmf10p30rgrELHcrMZuztQEIz+4eZndveMUnXokQhXdU2dz/Y3Q8AGoBvx/OiYJR9ezkeaDVRiCQTJQpJBm8Aw80sL5ir//1g7YGzAMzsEjObbmavAq+YWb6Z/d3MPg7m6z8nKHeymb1jZnODWkp+sH+Fmf0y2P+xmY226Pok3wZ+FNRsjo03WDMrNrNFZvaXYE2IF80sNzg2w8xuN7PZQZnDzOwxM/vMzG5u8R5PmNmc4PWTg33pQQ1gfhDnj3Y6b1pw/Oag7G1mNiv4HVwZlDEzu8ui6zu8DPTd+z+LJIuUHJktySOoIZwGPE90nqpX3f0yM+sJvB982EF0/v6x7r7ZzH5DdFqDA4P36BU0Cd0ATHD3GjO7FvgxcFPw+o3ufqiZfRf4qbt/y8z+DFS7++/2IvQRwIXufoWZPQScA9wXHGtw9xIzuxp4EhhHdHaApWZ2ezCS+rLgWnKBWWb2KFAMDAxqWQS/g+0ygPuJrmHw6yC5bHX3w8wsG3jLzF4EDgFGEV23oR+wEPjbXlyfJBElCumqcs3sw+D5G0SnXHkbOLNF30AOMDh4/pK7b5/DfwItpmRx9wozO4Poh+NbwRQmWcA7Lc73WPBzDnB2HPHtasqD7fuXu/v2+OcQ/ZDfbvv0Ih8DC7bP02Nmy4hO9rYJ+IGZTQrK7Us08XwCDDOz/wGeAV5s8Z53Aw+5+6+D7ZOBsS36H3oE7zEemObuEWBtUAuTFKdEIV3VNnc/uOUOi37Cn+Pun+y0/wigZjfvZ0STyYW7OL59Nt0I8f2/2QTsvFhMAbAl+Nlydt4IkNvKuZp3KtcMZJjZ8UST3VHuXmtmM4CcIOEdBJxCtFnsfOCy4LVvAyeY2e/dvY7o9V7l7i+0DNC6+DKhkhjqo5Bk8gJwVZAwts+I25qXgO9t3zCzXkRnjz3azIYH+/LMbORuzldF9EO/NTOJ1m4Kgvc7G5gXfFNvqx5ARZAkRhOdIn37bLBp7v4o0Wa0lmsl/5XoZHEPBc11LwDfMbPM4LUjzSwviPvfgj6MAUSXyJUUp0QhyeRXRGfR/MjMFgTbrbkZ6BV0+s4DTnD3cqKLGE0zs4+INjuN3s35ngImtdaZ7e4fAXcBbwZNZN8GvrV3l/UFzxOtWSwCbiWa5CC6itmM4Hz3AT/bKaY/EF0v+5/AVKL9D3PNbD7RpqkM4HHgs+DYvezY/CYpSrPHiohITKpRiIhITEoUIiISkxKFiIjEpEQhIiIxKVGIiEhMShQiIhKTEoWIiMT0/wFgtfl1E+9SrwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sequential_perturbation = benchmark.perturbation.SequentialPerturbation(\n",
    "    explainer.model, explainer.masker, sort_order, perturbation\n",
    ")\n",
    "xs, ys, auc = sequential_perturbation.model_score(shap_values, data[\"text\"][0:20])\n",
    "sequential_perturbation.plot(xs, ys, auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sort_order = \"negative\"\n",
    "perturbation = \"keep\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAq7klEQVR4nO3deXxU5b3H8c9vsu8bCUtCNvYdAQXctV7rVmnVeq22dWu1u1dv71Vr2+ut1l5vtdZqb1u3WmvFulCklqpYiyuCgILIvgQIW0KALGRPnvvHDBggywBJTmbm+369ziszZ87MfE8C85vzPOc8jznnEBGRyOPzOoCIiHhDBUBEJEKpAIiIRCgVABGRCKUCICISoaK9DnA0+vXr5woLC72OISISUpYsWbLbOZd9+PqQKgCFhYUsXrzY6xgiIiHFzDa3t15NQCIiEUoFQEQkQqkAiIhEqJDqAxCR0NDU1ERpaSn19fVeR4ko8fHx5OXlERMTE9T2KgAi0u1KS0tJSUmhsLAQM/M6TkRwzlFRUUFpaSlFRUVBPUdNQCLS7err68nKytKHfy8yM7Kyso7qqEsFQER6hD78e9/R/s4jogC8u343v5m/wesYIiJ9SkQUgDfXlnPfa2vYvq/O6ygi0otmz56NmbF69eqD6+bPn89FF110yHbXXHMNL7zwAuDvwL7tttsYNmwYkyZNYvr06fz9738/4rWvv/56JkyYwPjx47nsssuoqanpMMeWLVtITk7mvvvuO7juwQcfZOzYsYwZM4Zf/vKXRzzn/vvvx8zYvXs3AHv37uULX/gC48eP56STTmLFihVH9btoT0QUgK9OL8A5x1ML2r0YTkTC1MyZMzn11FOZOXNm0M/50Y9+xI4dO1ixYgVLly5l9uzZVFdXH7HdAw88wLJly1i+fDn5+fk8/PDDHb7mLbfcwvnnn3/w/ooVK3j00UdZtGgRy5Yt4+WXX2b9+vUHH9+6dSuvvfYa+fn5B9fdc889TJw4keXLl/PUU09x0003Bb1PHYmIApCXkchnxwxg5qIt1DY2ex1HRHpBTU0N77zzDo8//jjPPvtsUM+pra3l0Ucf5aGHHiIuLg6A/v37c/nllx+xbWpqKuA/+6aurq7D9vfZs2dTVFTEmDFjDq5btWoVU6dOJTExkejoaM444wxmzZp18PGbb76Z//3f/z3kNVeuXMnZZ58NwMiRIykpKWHXrl1B7VdHIuY00OtOLeLvK3bylw+3cdXUAq/jiESM//7rJ6zcXtWtrzl6UCr/9bkxnW7z0ksvcd555zF8+HCysrJYsmQJkydP7vQ569evJz8//+CHe1euvfZa5s6dy+jRo7n//vuPeLympoZ7772XefPmHdL8M3bsWO644w4qKipISEhg7ty5TJky5WDu3NxcJkyYcMhrTZgwgVmzZnHaaaexaNEiNm/eTGlpKf379w8qa3si4ggAYEpBBuNy03jinU20tmoeZJFwN3PmTK644goArrjiioPNQB19Uz+Ws5Z+//vfs337dkaNGsWf//znIx6/8847ufnmm0lOTj5k/ahRo7j11ls599xzOe+885g4cSJRUVHU1tZyzz338JOf/OSI17rtttvYt28fEydO5KGHHuKEE04gKirqqDMfwjkXMsvkyZPd8XhxyVZXcOvLbv6asuN6HRHp3MqVKz19/4qKCpeQkODy8/NdQUGBy8vLc4MHD3atra3u448/dieffPIh23/uc59z8+fPd/v373eZmZmusrLyqN7vzTffdBdeeOER60899VRXUFDgCgoKXFpamsvIyHAPPfTQEdvdfvvt7te//rVbvny5y87OPvicqKgoN3jwYLdjx45Dtm9tbXUFBQXt5mzvdw8sdu18pkbMEQDAheMHkp0SxxPvbPI6ioj0oBdeeIGvfOUrbN68mZKSErZu3UpRURFvv/02w4YNY/v27axatQqAzZs3s2zZMiZOnEhiYiLXX389N910E42NjQCUl5fz/PPPH/L6zrmDnbbOOebMmcPIkSOPyPH2229TUlJCSUkJ//Zv/8YPfvADvvOd7wBQVlYG+M8QmjVrFldeeSXjxo2jrKzs4HPy8vJYunQpAwYMYN++fQczPfbYY5x++ulBN1V1JGL6AADioqP4yrQCfjFvLevLahiak9z1k0Qk5MycOZNbb731kHWXXnopM2fO5PTTT+fpp5/m2muvpb6+npiYGB577DHS0tIAuPvuu/nhD3/I6NGjiY+PJykp6YgmGeccV199NVVVVTjnmDBhAr/5zW8AmDNnDosXL263GefwPBUVFcTExPDrX/+a9PT0TrdftWoVV199NWbGmDFjePzxx4/yt3Ik8x8dhIYpU6a4450QZndNAyf/7A0uPzGPuz8/rpuSiUhbq1atYtSoUV7HiEjt/e7NbIlzbsrh20ZUExBAv+Q4ZkwcxItLtlFZ2+R1HBERz0RcAQC49pQi6ppaePaDLV5HERHxTEQWgNGDUplWnMlTCzbT3NLqdRyRsBRKzcvh4mh/554UADP7uZmtNrPlZvYXM0vv7QzXnlLEtn11vLby+K6kE5EjxcfHU1FRoSLQi1xgPoD4+Pign+PVWUDzgNudc81mdi9wO3BrF8/pVueM6s/gzAR+/+4mLhg3sDffWiTs5eXlUVpaSnl5uddRIsqBGcGC5UkBcM691ubu+8BlvZ0hymdcPb2Qu/+2ihXbKhmbm9bbEUTCVkxMTNCzUol3+kIfwHXAkWOtBpjZDWa22MwWd/e3iS9OGUxCTBR/eK+kW19XRCQU9FgBMLPXzWxFO8uMNtvcATQDf+rodZxzjzjnpjjnpmRnZ3drxrSEGC6ZlMtLy7azZ39jt762iEhf12MFwDl3jnNubDvLSwBmdg1wEXCV87Cn6OqTC2lsbuXPH2z1KoKIiCe8OgvoPOA/gYudc7VeZDhgeP8Uphdn8fT7OiVURCKLV30ADwMpwDwz+8jMfutRDsB/FLBtXx2vryrzMoaISK/y6iygoV68b0fOGZVDbnoCTy0o4byxA7yOIyLSK/rCWUCei47ycdW0fN7bUMG6XUfO/SkiEo5UAAL+dcpgYqKMZxZpfCARiQwqAAFZyXGcN3YgLy4ppb6pxes4IiI9TgWgjStPyqeqvpmXl+/wOoqISI9TAWhjWnEmxdlJPLNws9dRRER6nApAG2bGlSfls3TLPlbtqPI6johIj1IBOMxlk/OIjfbxzEJ1BotIeFMBOEx6YiwXjRvIXz7cxv6GZq/jiIj0GBWAdlx+4mBqGpp5c63GMheR8KUC0I4pBRmkxEfz5hoVABEJXyoA7YiO8nHq0H68ubZcU9qJSNhSAejAmSOy2VlVz9pdNV5HERHpESoAHTh9uH/ymflrNEKoiIQnFYAODExLYET/FHUEi0jYUgHoxJkjsvmgZI9OBxWRsKQC0IkzhmfT1OJ4b0OF11FERLqdCkAnphRmkhgbxZtr1Q8gIuFHBaATsdE+Th7Sj/lrdDqoiIQfFYAunDEim9K9dWzcvd/rKCIi3UoFoAtnDPOfDvru+t0eJxER6V4qAF0YnJlAbnoCC9QRLCJhRgWgC2bG9CFZLNhYQWur+gFEJHyoAARhenEW+2qbWL2z2usoIiLdRgUgCNOHZAGwYKOagUQkfKgABGFQegKFWYks2KCOYBEJHyoAQZo+JIuFm/bQon4AEQkTKgBBmlacRXV9M59sr/Q6iohIt1ABCNKBfgCNCyQi4UIFIEg5KfEMzUnW9QAiEjZUAI7C9OIsPijZQ1NLq9dRRESOmwrAUTh5SBa1jS0sL93ndRQRkeOmAnAUphb7+wHe37jH4yQiIsdPBeAoZCbFMnJACu/rgjARCQMqAEdpalEmi0v2qh9AREKeCsBRmlacRV1TC8tLdT2AiIQ2FYCjdFJRJgALN6kZSERCmwrAUcpKjmN4/2R1BItIyPOkAJjZXWa23Mw+MrPXzGyQFzmO1dSiLJboegARCXFeHQH83Dk33jk3EXgZ+LFHOY7JtOIs9je2sGKb+gFEJHR5UgCcc1Vt7iYBITXE5tRifz+AmoFEJJR51gdgZj81s63AVXRyBGBmN5jZYjNbXF5e3nsBO9EvOY6hOcnqCBaRkNZjBcDMXjezFe0sMwCcc3c45wYDfwK+09HrOOcecc5Ncc5Nyc7O7qm4R21acSYfbNpDs/oBRCRE9VgBcM6d45wb287y0mGb/gm4tKdy9JSpRYF+gO1VXW8sItIHeXUW0LA2d2cAq73IcTwO9AO8p2kiRSREedUH8D+B5qDlwLnATR7lOGY5KfGMHpjKP1eXeR1FROSYRHvxps65kGvyac/ZI3P4v/nr2VfbSHpirNdxRESOSpdHAGb2xWDWRaKzRubQ6uDNtX3j7CQRkaMRTBPQ7UGuizgTB6eTmRSrZiARCUkdNgGZ2fnABUCumf2qzUOpQHNPBwsFUT7jjOHZzF9TRkurI8pnXkcSEQlaZ0cA24HFQD2wpM0yB/hsz0cLDWeNzGFvbRMfbd3rdRQRkaPS4RGAc24ZsMzM/uSc0zf+DpwxLJson/HG6jImF2R6HUdEJGjB9AGsM7ONhy89nixEpCXGMDk/gzdWqyNYREJLMKeBTmlzOx74IqCvum2cNTKHe19ZzY7KOgamJXgdR0QkKF0eATjnKtos25xzvwQu7PlooePskTkAvKGzgUQkhHR5BGBmk9rc9eE/IvDkArK+anj/ZIqzk3hhSSlXTS3wOo6ISFCC+SC/v83tZqAEuLxH0oQoM+OqqQXc9fJKVmyrZGxumteRRES6FEwT0Fltln9xzn3dObemN8KFkssm5REf4+NPCzd7HUVEJCjBDAWRZWa/MrOlZrbEzB40s6zeCBdK0hJjuHjCIGZ/uJ2q+iav44iIdCmY00CfBcrxj9l/WeD2n3syVKj68rQC6ppa+MvSbV5HERHpUjAFYKBz7i7n3KbAcjfQv6eDhaLxeemMz0vj6fc341xITXMsIhEomALwmpldYWa+wHI58GpPBwtVX55awLqyGhZt0oTxItK3BVMAvg48AzQElmeBG82s2sw0H+JhPjdhEKnx0fxhQYnXUUREOhXMWUApzjmfcy4msPgC61Kcc6m9ETKUJMRGceXUAl5ZsZOte2q9jiMi0qFgzgL6RzDr5FPXnFyIz4zH39nkdRQRkQ51WADMLN7MMoF+ZpZhZpmBpRDI7bWEIWhAWjwXTxjEc4u3UlmrU0JFpG/q7AjgRvzj/48ElvLpfAAvAQ/3fLTQ9rXTiqltbOGZRVu8jiIi0q4OC4Bz7kHnXBHwfedcUZtlgnNOBaALowelcurQfjz53iYam1u9jiMicoRgzgKqNLOvHr70eLIw8LXTithV1cBfl233OoqIyBGCKQAntllOA+4ELu7BTGHjjOHZDO+fzBPvbtKFYSLS53Q5Gqhz7rtt75tZOv5rAaQLZsZXpxfyw9krWLplH5MLMryOJCJyUDBHAIfbDxR1d5Bw9YUTckmJi+YpXRgmIn1MMNcB/NXM5gSWl4E1wF96Plp4SIqL5rIpecz9eAdl1fVexxEROSiYCWHua3O7GdjsnCvtoTxh6SvTCvj9uyU8u2gr3/vMMK/jiIgAwTUBbQFSAssOffgfveLsZE4fns2fFm6mqUWnhIpI39DZlcCpZvYc8DpwXWB53cyeNzONAXSUrp5ewK6qBl77ZJfXUUREgM6PAH4FrASGOecucc5dAgwBPkZXAh+1M0fkMDgzgcfe2ahTQkWkT+isAJzinLvTOXewzcL5/QSY3vPRwkuUz7jx9CF8uGUfb64t9zqOiMgxnQYKYN2aIkJcPmUwuekJ/GLeWh0FiIjnOisA75nZj83skA97M/sRsKBnY4Wn2Ggf3/vMUJaXVvL6qjKv44hIhOusAHwXGAesN7MXA8sGYALwnV5JF4YumZRHQVYiv5i3ltZWHQWIiHc6Gw20yjn3ReBc4MnAcq5z7jLnXGXvxAs/MVE+vnf2MFbtqOLVT3Z6HUdEIlgwU0JucM79NbBs6I1Q4e7zJ+RSnJ3E/fPW0qzrAkTEI8faCSzHIcpn/OdnR7K+rIaZmjBGRDziaQEws383M2dm/bzM4YXPjunP1KJMfjFvLZV1mjZSRHpfZ1cCZ3a2HO8bm9lg/P0LEfkV2Mz40UWj2VfXxMNvrPM6johEoM4Gg1sCONo/598Bxcf53g8A/4l/juGINDY3jS9OzuPJ90q4amoBhf2SvI4kIhGks7OAipxzxYfNB3xgOa4PfzObAWxzzi0LYtsbzGyxmS0uLw+/K2i/f+4IYqN83PXySl0cJiK9Kpj5AMzMvhy4AAwzyzezk4J43utmtqKdZQbwA+DHwQR0zj3inJvinJuSnZ0dzFNCSk5qPLecO4J/rC7jl6+rKUhEek8w8wH8H9AKnA3cBVQDL+KfI7hDzrlz2ltvZuPwzyi2LHCRcR6w1MxOcs5F5Inx151SyOodVTz4j3UUZCVyyaQ8ryOJSAQIpgBMdc5NMrMPAZxze80s9ljf0Dn3MZBz4L6ZlQBTnHO7j/U1Q52Z8dMvjKN0bx23vric3PQEphZneR1LRMJcMKeBNplZFP6OX8wsG/8RgXSj2Ggfv/3yZPIzE/n6U4tZXrrP60giEuaCKQC/wj8HcI6Z/RR4B7inuwI45woj+dt/W2mJMfzhupNIS4zhqscW8uGWvV5HEpEwFsxQEH/Cf7rmz4AdwOedc8/3dLBIlZeRyLM3TCcjMZavPr6IJZtVBESkZwR1IRhQBswEngF2dceFYNKx3PQE/nzjNLKSY7nmiUVs3VPrdSQRCUOdHQEsARYHfpYDa4F1gdtLej5aZBuYlsAfr59Kq3PcNmu5rhEQkW7X5YVg+CeF/5xzrp9zLgu4CHittwJGssGZidx+wSjeXV/BzEVbvY4jImEmmE7gac65uQfuOOf+Dpzcc5GkrStPyufkIVncM3cV2/bVeR1HRMJIMAVgu5n90MwKA8sdwPaeDiZ+Pp9x76Xj/U1BL6opSES6TzAF4EtANv5TQf+C/yKuL/VkKDnU4MxEbjt/JG+v283fPt7hdRwRCRNdXgnsnNsD3GRmKf67rqbnY8nhrppawMxFW/nZ3NWcM6o/8TFRXkcSkRAXzGBw4wLDQKwAPjGzJWY2tuejSVtRPuO/PjeabfvqeOStjV7HEZEwEEwT0O+AW5xzBc65AuDfgUd6Npa0Z1pxFheMG8D/zV/PdnUIi8hxCqYAJDnn/nngjnNuPqCZSzxy+/mjaHVw7yurvY4iIiEumAKw0cx+1OYsoB8CaoPwyODMRG48vZiXPtrOwo0VXscRkRAWTAG4Dv9ZQLMCS3ZgnXjkW2cOZXBmArfP+pj6phav44hIiApmMLi9zrnvOecmBZabnHMaocxDCbFR3POFcWzcvZ+H31jvdRwRCVEdngZqZnM6e6Jz7uLujyPBOm1YNpdMyuW3b27gwvEDGTUw1etIIhJiOrsOYDqwFf8ooAsB65VEErQfXTiaN9eUc9usj5n1zZOJ8ulPJCLB66wJaAD+ydvHAg8C/wLsds696Zx7szfCSecykmL58edGs2zrPmYu2uJ1HBEJMZ2NBtrinHvFOXc1MA1YD8w3s+/0Wjrp0sUTBnFSUSa/mLeWqvomr+OISAjptBPYzOLM7BLgaeDbfDo9pPQRZsaPLxrN3tpGHvrHOq/jiEgI6WxGsKeABcAk4L+dcyc65+5yzm3rtXQSlLG5aXxxch5PvlfCpt37vY4jIiGisyOALwPDgJuA98ysKrBUm1lV78STYH3/syOIjfJxz9xVXkcRkRDRWR+AzzmXElhS2ywpzjmdc9jH5KTE8+2zhzJv5S7eWbfb6zgiEgKCuRJYQsR1pxSRn5nInX/9hKaWVq/jiEgfpwIQRuJjovjRRaNZX1bDUws2ex1HRPo4FYAwc86oHE4fns0v562lvLrB6zgi0oepAIQZM//EMfXNLfyvhowWkU6oAIShIdnJXHdKEc8vKWXJ5j1exxGRPkoFIEx99zPDyE1P4KZnP6KyVlcIi8iRVADCVHJcNA9feQI7K+v5jxeW4ZzzOpKI9DEqAGHshPwMbjt/JK+t3MWT75V4HUdE+hgVgDB3/alFnDMqh3vmruKjrfu8jiMifYgKQJgzM+774gT6p8bztT8sZuueWq8jiUgfoQIQAdITY3ny2hNpbG7h2ic/UKewiAAqABFjaE4Kj3x1Clsqarnx6cU0NGsyeZFIpwIQQaYVZ/HzL47n/Y17+PfnltHSqjODRCJZZ3MCSxiaMTGXsqoGfjp3Fclx0fzsknGYaS5hkUikAhCBvn56MVX1TTz0xnqS46K548JRKgIiEciTAmBmdwJfB8oDq37gnJvrRZZIdcu/DKe6vpnH3tlElM+49byR+HwqAiKRxMsjgAecc/d5+P4R7cBcws2trfzurY1sKK/hgX+dSEp8jNfRRKSXqBM4gvl8xl0zxvKTGWP455pyLvm/91hfVu11LBHpJV4WgO+Y2XIze8LMMjrayMxuMLPFZra4vLy8o83kGJkZX51eyB+vO4nymgbO+cVbXP3EIl5fuUtnCYmEOeupQcLM7HVgQDsP3QG8D+wGHHAXMNA5d11XrzllyhS3ePHibs0pnyqrrueZhVuYuWgLu6oa6JccyxnDc/jMqBxOGdKPtEQ1D4mEIjNb4pybcsR6r0eJNLNC4GXn3NiutlUB6B1NLa38Y9Uu5n68k/lryqiqbwZgcGYCYwamMTE/nTNHZDOif4rOHhIJAX2qAJjZQOfcjsDtm4GpzrkrunqeCkDva25pZemWfSzevIdPtlXxyfZKSir84wnlpifwL6P7c/2pRQzOTPQ4qYh0pK8VgD8CE/E3AZUANx4oCJ1RAegbdlbW8881Zbyxuow315TT4hwzJg7iW2cOZWhOstfxROQwfaoAHCsVgL5nZ2U9j7y1kWcWbaahuZVzR/fnG2cM4YT8Dvv1RaSXqQBIj6qoaeDJ90p4asFmKuuamFqUyQ2nF3PWiBxdYCbiMRUA6RU1Dc08u2gLj7+ziR2V9RRnJ3H9qUV8fmIuSXEaeUTECyoA0quaWlqZ+/EOHn17Iyu2VZEUG8XFEwfxpZPyGZ+X7nU8kYiiAiCecM6xZPNenv1gKy8v3059UytTCjK48YwhfGakmodEeoMKgHiuqr6JF5eU8tjbm9i2r44h2UlcObWAGRMH0S85zut4ImFLBUD6jOaWVv728Q4ef2cTy0srifYZZ47I4dJJuZw9Koe46CivI4qElY4KgHrlpNdFR/mYMTGXGRNzWburmheXlDLrw228vmoXqfHRXDh+IFecmM+EweleRxUJazoCkD6huaWVdzdUMPvDbbyyYid1TS1MGJzO1dMLuHD8QB0ViBwHNQFJyKiub2LW0m38YUEJG8v3Mygtnu99ZhiXTs4jJkojmIscLRUACTnOOd5at5sH5q3lo637KMxK5D8+O5ILxg3QIHQiR6GjAqCvU9JnmRlnDM/mL986mce+OoX4mCi+/cxSrnpsIet2aeIakeOlAiB9nplxzuj+/O17p3HX58fyyfYqzn/wbX42dxV1jS1exxMJWSoAEjKifMZXphXwz++fyWWT8/jdWxs578G3eG/Dbq+jiYQkFQAJOZlJsfzPpeOZ+fVpGHDlowu55bmP2Lqn1utoIiFFBUBC1vQhWbzyb6fzjTOG8PLyHZx133xue3E5m3bv9zqaSEjQWUASFnZW1vOb+euZuWgrjS2t5KYnMK04i6lFmUwuzKC4X5LOHJKIpdNAJSLsqKzj1RU7eX/jHhZuqmBvbRMAGYkxTMrPYEphJicWZjAuL00Xl0nEUAGQiNPa6thQXsPSLXtZsnkvizfvZWO5v3koLtrHpPwMphVnMa04kwmD04mPUUGQ8KQCIIJ/5rLFm/eyaNMe3t9YwcodVTgHsdE+JuWnM604i1OG9mPi4HRddSxhQwVApB2VtU0s3FTBwk3+JqNPtvsLQlJsFFOLszhzRDZnj8whLyPR66gix0wFQCQIlbVNLNhYwbvrd/PWunI2V/hPLR05IIULxg1kxsRBFGQleZxS5OioAIgcgw3lNbyxqox5K3exqGQPACfkp/P5iblcNH4gWZrIRkKACoDIcdq+r445y7Yz+8NtrN5ZTZTPP1bReWMHcObwbHJS472OKNIuFQCRbrR6ZxWzP9zOnI+2sb2yHoAxg1I5eUgWk/IzOCE/gwFpKgjSN6gAiPQA5xyrdlTzzzVlzF9TxrKtlTS2tALQPzWOcbnpjMtNY8SAZHLTE8nNSCAjMUYXpUmvUgEQ6QUNzS2s2lHN0s17+XhbJctL97Fx937a/jeLjfKRmhBNanwMWcmxFGYlUZSdRGFWEoMzEhmcmUBagoqEdB/NCSzSC+Kio5g4OJ2JbeYzrmlopmT3frbtq2Pb3jrKqhuorm+isq6JsuoG3lxbzvNLSg95neS4aPqnxjEgLZ7+KfGkJ8aSkRhDemIMCbHRJMREER/jIz4mithoH3HRPmKjfcRGBX5G+4iL+vQxn0/FRI6kAiDSw5Ljohmbm8bY3LQOt6lpaGZzxX627qmjdG8tpXvr2FlZz67qehZu2sO+2kb2H8fcB7HRPhJiooiJ8hHlg2ifjyifEe0zfIGfsdE+YqJ8JMZGkZbgLzYZibFkp8SRkxJHZlIcSXFRJMdFkxQXTXJcNHHRPh2phDAVAJE+IDkumjGD0hgzqOMi0djcSmVdE/VNLdQ1tVDb2EJjcyuNza00NAdut7TS0NRKQ0vrwcfqm1qob26hvrGFplZHS4ujudXR6vw/m1taD/5sanFU1zezbW8d++qa2FfbSGsnrcRRPiMpNoqE2CgSY6OJj4kiJS6a5PhoUuKjyUiMJTPJv/RLjqVfchzZKXEMSk/QldZ9gAqASIiIjfaRndK71x20tDr27G+kvLqBPfsbqWlopqahmf2Bn7WNzexvaDlYlPY3tLC/oZmy6no2lDezZ38j1fXNR7xutM8o7JfEsJxkhuUkM7R/CsNykinql6QxmXqRCoCIdCjKZ2SnxB1X4WlsbmVvbSO7axoor26grLqBTbv3s76shtU7q3n1k50HjzJ8BgVZ/sIwNCeZIdn+n0NzkkmK08dVd9NvVER6VGy0j/6p8fTv4EK5+qYWSir2s3ZXDevLali3q5q1u6p5Y3UZzW3anwZnJjCifwrD+qccLAxDspNIiY/prV0JOyoAIuKp+JgoRg5IZeSA1EPWN7W0srmilvVlNawvq2b1zmrW7Kxm/pryQwpD/9S4gwVh1MBURg1MZUT/FBJi1ZTUFRUAEemTYqJ8B5t/YMDB9U0trWzZU8uGshrWl9ewsXw/G8prmLV0GzUNmwEwg/zMRIb3T2FE/xSGD/D/LOqXRGy0Op8PUAEQkZASE+VjSLa/f+DcNutbWx2le+tYuaOK1TurWLerhjWBpqSWwBFDtM8YmpPMyAEpjByYyuiBqYwelEq/CB3UTwVARMKCz2fkZyWSn5XIeWM/PWJoaG5hY/l+1u76tBlp0aY9zP5o+8FtclLiGD0olTGDUgOn46YyOCMx7C+gUwEQkbAWFx11sG9gRpv1+2obWbmjipXbA8uOKt5et/vg0UJS7KfPGz0oPPsWPBsLyMy+C3wbaAH+5pz7z66eo7GARKQn1Te1sHZXNat2fFoUVu2opqbBfy2DGRRlJTFyYApDs5Mp7JdEQVYSuekJZCXH9tmL2/rUWEBmdhYwA5jgnGswsxwvcoiItBUfE8X4vHTG56UfXHd438LqHdV8sr2KV1bsPOIq6fTEGJLjog8OsWEQuMraf+W1f4Fj+d79qysmcvLQfse1f4fzqgnom8D/OOcaAJxzZR7lEBHpVEd9C43NrWzdW0vJ7v3srKpnd7X/Yrf9Dc20BIbZwEF0lBHt8+Ez/4V1ZsaxDJ/UrweuAveqAAwHTjOznwL1wPedcx+0t6GZ3QDcAJCfn997CUVEOhEb/enZSKGqxwqAmb1O25N3P3VH4H0zgWnAicBzZlbs2umQcM49AjwC/j6AnsorIhJpeqwAOOfO6egxM/smMCvwgb/IzFqBfkB5T+UREZFDedVlPRs4C8DMhgOxwG6PsoiIRCSv+gCeAJ4wsxVAI3B1e80/IiLSczwpAM65RuDLXry3iIj49c2rFkREpMepAIiIRCgVABGRCOXZWEDHwszKgc3H+PR+RN6ZRtrnyKB9jgzHs88Fzrnsw1eGVAE4Hma2uL3BkMKZ9jkyaJ8jQ0/ss5qAREQilAqAiEiEiqQC8IjXATygfY4M2ufI0O37HDF9ACIicqhIOgIQEZE2VABERCJU2BUAMzvPzNaY2Xozu62dx+PM7M+BxxeaWaEHMbtVEPt8i5mtNLPlZvYPMyvwImd36mqf22x3qZk5MwvpUwaD2V8zuzzwd/7EzJ7p7YzdLYh/1/lm9k8z+zDwb/sCL3J2JzN7wszKAgNltve4mdmvAr+T5WY26bje0DkXNgsQBWwAivEPMb0MGH3YNt8Cfhu4fQXwZ69z98I+nwUkBm5/MxL2ObBdCvAW8D4wxevcPfw3HgZ8CGQE7ud4nbsX9vkR4JuB26OBEq9zd8N+nw5MAlZ08PgFwN8Bwz+h1sLjeb9wOwI4CVjvnNvo/COOPot/8vm2ZgB/CNx+AfiM2bHM0NlndLnPzrl/OudqA3ffB/J6OWN3C+bvDHAXcC/+aUdDWTD7+3Xg1865vRAW82wHs88OSA3cTgO292K+HuGcewvY08kmM4CnnN/7QLqZDTzW9wu3ApALbG1zvzSwrt1tnHPNQCWQ1SvpekYw+9zW9fi/QYSyLvc5cGg82Dn3t94M1kOC+RsPB4ab2btm9r6Znddr6XpGMPt8J/BlMysF5gLf7Z1onjra/++d8mpCGPGAmX0ZmAKc4XWWnmRmPuAXwDUeR+lN0fibgc7Ef4T3lpmNc87t8zJUD/sS8KRz7n4zmw780czGOudavQ4WKsLtCGAbMLjN/bzAuna3MbNo/IeOFb2SrmcEs8+Y2TnAHcDFzrmGXsrWU7ra5xRgLDDfzErwt5XOCeGO4GD+xqXAHOdck3NuE7AWf0EIVcHs8/XAcwDOuQVAPP4B08JZUP/fgxVuBeADYJiZFZlZLP5O3jmHbTMHuDpw+zLgDRfoXQlRXe6zmZ0A/A7/h3+otw1DF/vsnKt0zvVzzhU65wrx93tc7Jxb7E3c4xbMv+vZ+L/9Y2b98DcJbezFjN0tmH3eAnwGwMxG4S8A5b2asvfNAb4aOBtoGlDpnNtxrC8WVk1AzrlmM/sO8Cr+swiecM59YmY/ARY75+YAj+M/VFyPv7PlCu8SH78g9/nnQDLwfKC/e4tz7mLPQh+nIPc5bAS5v68C55rZSqAF+A/nXMge2Qa5z/8OPGpmN+PvEL4mxL/MYWYz8RfyfoG+jf8CYgCcc7/F39dxAbAeqAWuPa73C/Hfl4iIHKNwawISEZEgqQCIiEQoFQARkQilAiAiEqFUAEREIpQKgPQ5ZtZiZh+Z2Qoze97MEj3IcKaZndzBY3ea2fcPW1cSOP++V5nZ/GO9wM3MnjSzy7o7k4QOFQDpi+qccxOdc2OBRuAbwTwpcGV3dzkTaLcAiIQLFQDp694GhppZUmCs9EWB8d9nAJjZNWY2x8zeAP5hZslm9nsz+zgwXvqlge3ONbMFZrY0cFSRHFhfYmb/HVj/sZmNNP8cEd8Abg4ciZwWbFgzKzSzVWb2aGBc/tfMLCHw2Hwze8DMFge2OdHMZpnZOjO7u81rzDazJYHn3xBYFxX4xr4ikPPmw97XF3j87sC2PzezDwK/gxsD25iZPWz+MfZfB3KO/c8i4SCsrgSW8BL4Rn8+8Ar+cYzecM5dZ2bpwKLAhxj4x08f75zbY2b34r88flzgNTICTTM/BM5xzu03s1uBW4CfBJ6/2zk3ycy+BXzfOfc1M/stUOOcu+8Yog8DvuSc+7qZPQdcCjwdeKzROTfFzG4CXgIm478ifYOZPRC4eve6wL4kAB+Y2YtAIZAbOCoi8Ds4IBr4E/4x5H8aKBqVzrkTzSwOeNfMXgNOAEbgHzu/P7ASeOIY9k/ChAqA9EUJZvZR4Pbb+IfveA+4uE3bezyQH7g9zzl3YAz1c2gzvIdzbq+ZXYT/Q+/dwFAYscCCNu83K/BzCXBJEPk6unz+wPpNzrkD+Zfg//A+4MAwFR8DnxwYx8XMNuIf5KsC+J6ZfSGw3WD8BWUNUGxmDwF/A15r85q/A55zzv00cP9cYHyb9v20wGucDsx0zrUA2wNHTRLBVACkL6pzzk1su8L8n9yXOufWHLZ+KrC/i9cz/EXiSx08fmB01BaC+z9RARw+CUcKsC/ws+1oqy1AQjvv1XrYdq1AtJmdib+ITXfO1ZrZfCA+UMgmAJ/F3zx1OXBd4LnvAWeZ2f3OuXr8+/td59yrbQNaGEyZKN1LfQASKl4FvhsoBAdGOG3PPODbB+6YWQb+0UBPMbOhgXVJZja8i/erxv9h3p638B+NpARe7xJgWeCb9fFKA/YGPvxH4h/K+sAInz7n3Iv4m7PazgX7OP5Bwp4LNJu9CnzTzGICzx1uZkmB3P8a6CMYiH+qUIlgKgASKu7CPyricjP7JHC/PXcDGYHO0mXAWc65cvyTw8w0s+X4m39GdvF+fwW+0F4nsHNuOfAw8E6gqeobwNeObbeO8Ar+I4FVwP/gL17gn/VpfuD9ngZuPyzTL/DPCfxH4DH87ftLzT+5+O/wH9n8BVgXeOwpDm0Gkwik0UBFRCKUjgBERCKUCoCISIRSARARiVAqACIiEUoFQEQkQqkAiIhEKBUAEZEI9f800OBvwrCUvwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sequential_perturbation = benchmark.perturbation.SequentialPerturbation(\n",
    "    explainer.model, explainer.masker, sort_order, perturbation\n",
    ")\n",
    "xs, ys, auc = sequential_perturbation.model_score(shap_values, data[\"text\"][0:20])\n",
    "sequential_perturbation.plot(xs, ys, auc)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}